# -*- coding:utf-8 -*-
import ctypes
import threading
import MySQLdb
import os
import socket
import json
import httplib
import urllib2
import uuid
import zipfile
import base64
import sys
import hashlib
import datetime
import time
import subprocess
from bs4 import BeautifulSoup
import StringIO
import pycurl
from urlparse import urlparse
import pickle
import phpserialize
import random

#http://blog.wanthings.com/archives/239

'''
convert a ISO format time to second
from:2006-04-12 16:46:40 to:23123123
把一个时间转化为秒
'''
def ISOString2Time(s,ISOTIMEFORMAT):
    return time.strptime(s, ISOTIMEFORMAT)

def commom_log(filename, content , write_type='wb'):
    try:
        f = open(filename , write_type)
        f.write(content);
        f.close()
    except:
        print('happen an exception when write file')
        return ""

def initCurl():
    c = pycurl.Curl()
    c.setopt(pycurl.COOKIEFILE, "cookie_file_name")
    c.setopt(pycurl.COOKIEJAR, "cookie_file_name")
    c.setopt(pycurl.FOLLOWLOCATION, 1)
    c.setopt(pycurl.MAXREDIRS, 5)
    return c

def GetData(curl, url):
	ip_str = str(random.randint(1 , 255)) + "." + str(random.randint(1 , 255)) + "." + str(random.randint(1 , 255)) + "." + str(random.randint(1 , 255))
	head = ['Accept:*/*',
			"X-FORWARDED-FOR:" + ip_str,
			"CLIENT-IP:" + ip_str,
            "Content-Type:text/html; charset=UTF-8",
            "Host:itunes.apple.com",
            'Accept-Charset:GBK,utf-8;q=0.7,*;q=0.3',
            'Accept-Language:zh-CN,zh;q=0.8',
			'User-Agent:Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11']
	buf  = StringIO.StringIO()
	curl.setopt(pycurl.WRITEFUNCTION,buf.write)
	curl.setopt(pycurl.URL,url)
	curl.setopt(pycurl.REFERER,"https://itunes.apple.com")
	curl.setopt(pycurl.HTTPHEADER,head)
	curl.setopt(pycurl.SSL_VERIFYPEER,False)
	curl.setopt(pycurl.TIMEOUT , 6000)
	curl.setopt(pycurl.CONNECTTIMEOUT, 60000)
	curl.setopt(pycurl.MAXREDIRS, 5)
	# curl.setopt(pycurl.HTTP_VERSION,'1.0')

	curl.perform()
	the_page =buf.getvalue()
	buf.close()
	print str(curl.getinfo(pycurl.HTTP_CODE)) + "------------------->>>>>>>>"
	return the_page

def PostData(curl, url, data):  
	ip_str = str(random.randint(1 , 255)) + "." + str(random.randint(1 , 255)) + "." + str(random.randint(1 , 255)) + "." + str(random.randint(1 , 255))
	head   = ['Accept:*/*',
	"X-FORWARDED-FOR:" + ip_str,
	"CLIENT-IP:" + ip_str,
	"Host:itunes.apple.com",
	'Content-Type:application/xml',
	'render:json',
	'clientType:json',
	'Accept-Charset:GBK,utf-8;q=0.7,*;q=0.3',
	'Accept-Encoding:gzip,deflate,sdch',
	'Accept-Language:zh-CN,zh;q=0.8',
	'User-Agent:Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11']

	buf = StringIO.StringIO()
	curl.setopt(pycurl.WRITEFUNCTION, buf.write)
	curl.setopt(pycurl.POSTFIELDS,  data)
	curl.setopt(pycurl.URL, url)
	curl.setopt(pycurl.HTTPHEADER,  head)
	curl.setopt(pycurl.TIMEOUT , 6000)
	curl.perform()
	the_page = buf.getvalue()
	buf.close()
	return the_page

def get_itunesid(url):
	url = urlparse(url)
	return os.path.basename(url.path).replace("id" , "")

def ___parsedata(html_str , title , a_link , itunesid , cul_typeid , cul_name , cul_pid , cul_pname):
	soup = BeautifulSoup(html_str)
	dict_map = dict()
	if soup.find(id="content"):
		content_html = soup.find(id="content")
		center_stack = content_html.find('div' , {'class' : 'center-stack'})
		left_stack   = content_html.find(id="left-stack")
		if left_stack:
			ul_html = left_stack.find("div" , {'class' : "lockup"}).find("ul","list")

			dict_map.setdefault("itunesid" , itunesid)
			dict_map.setdefault("title" , title)
			dict_map.setdefault("apptypesno" , cul_name)
			dict_map.setdefault('fatherid' , cul_pid)
			dict_map.setdefault("childid" , cul_typeid)
			dict_map.setdefault("itunes" , a_link)

			log_img = left_stack.find("div","lockup").find("a").find("div" , 'artwork').find("img" , "artwork").get("src")
			#得分
			score = 0
			if left_stack.findAll("span" , 'rating-count'):

				if len(left_stack.findAll("span" , 'rating-count')) > 1:
					score = left_stack.findAll("span" , 'rating-count')[1].string
				else:
					score = left_stack.findAll("span" , 'rating-count')[0].string

				score = score.encode("utf-8")
				score = filter(str.isdigit,score)

			dict_map.setdefault("score" , score)


			#时间
			senddate = ul_html.find("li" , "release-date").getText()
			senddate = senddate.encode("utf-8")
			senddate = filter(str.isdigit,senddate)
			senddate = time.strftime('%Y-%m-%d', ISOString2Time(str(senddate),'%Y%m%d'))

			dict_map.setdefault("senddate" , senddate)

			#版本
			version = ul_html.findAll("li")[3].getText()
			version = version.encode("utf-8")
			version = version.replace("版本:","").strip()

			dict_map.setdefault("version" , version)

			#语言
			if ul_html.find('li' , 'language'):
				language = ul_html.find('li' , 'language').getText()
				if language:
					language = language.encode("utf-8")
					language = language.replace("语言:","").strip()
				else:
					language = '未知'
			else:
				language = '未知'

			dict_map.setdefault("language" , language)

			#app 价格
			itunesprice = ul_html.findAll("li")[0].getText()
			itunesprice = itunesprice.encode("utf-8")
			if itunesprice == '免费':
				itunesprice = 0
			else:
				itunesprice = itunesprice.replace("," , "").strip()
				itunesprice = itunesprice.replace("¥" , "").strip()

			dict_map.setdefault("itunesprice" , itunesprice)

			#大小 | 尺寸
			size = ul_html.findAll("li")[4].getText()
			size = size.encode("utf-8")
			size = size.replace("大小：","").strip()
			if size.find("GB") != -1:
				size = size.replace("GB" , "")
				size = float(size) * 1024
			else:
				size = size.replace("MB" , "")
				size = float(size)

			dict_map.setdefault("size" , size)

			#更新日期
			appupdate = senddate
			dict_map.setdefault("appupdate" , appupdate)

			#系统说明
			system_m = ul_html

			dict_map.setdefault("system_m" , system_m)

			#是否免费
			if itunesprice == 0:
				isopen = 2
			else:
				isopen = 1

			dict_map.setdefault("isopen" , isopen)

			# 开放商
			seller = ul_html.findAll("li")[6].getText()
			seller = seller.encode("utf-8").strip()
			
			dict_map.setdefault("seller" , seller)

			#系统要求
			system = left_stack.find("div",'lockup').find("p")

			dict_map.setdefault("system" , system)

			#设备类型
			equipmenttype = 1
			if system.find("iPhone") != -1:
				if system.find("iPad") != -1:
					equipmenttype = 3
			else:
				equipmenttype = 2

			dict_map.setdefault("equipmenttype" , equipmenttype)

			iphoneimg = list()
			ipadimg   = list()

			iphoneimg_html = soup.find("div" , 'iphone-screen-shots')
			if iphoneimg_html:
				for img in iphoneimg_html.findAll("div" , "lockup"):
					iphoneimg.append(img.find("img").get("src"))

			iphoneimg = phpserialize.dumps(iphoneimg)

			dict_map.setdefault("iphoneimg" , iphoneimg)

			ipadimg_html = soup.find("div" , "ipad-screen-shots")
			if ipadimg_html:
				for img in ipadimg_html.findAll("div" , "lockup"):
					ipadimg.append(img.find("img").get("src"))

			ipadimg = phpserialize.dumps(ipadimg)

			dict_map.setdefault("ipadimg" , ipadimg)

			content = center_stack.find("div","product-review")
			if content:

				content = content.contents
				content_new = ""
				for aa in content:
					content_new += aa.encode("utf-8")

				content = content_new
				content = content.replace("height: 54px;","")
				content = content.replace("<a class=\"more-link\" href=\"#\">...更多</a>" , "");

				dict_map.setdefault("content" , content)

				newfeatures = ''
				if len(center_stack.findAll("div","product-review")) > 1:
					newfeatures = center_stack.findAll("div","product-review")[1]
					newfeatures = newfeatures.contents

					newfeatures_new = ""
					for aa in newfeatures:
						newfeatures_new += aa.encode("utf-8")

					newfeatures = newfeatures_new

					newfeatures = newfeatures.replace("height: 54px;" , "")
					newfeatures = newfeatures.replace("<a class=\"more-link\" href=\"#\">...更多</a>" , "");

				dict_map.setdefault("newfeatures" , newfeatures)

				getdatetime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
				
				dict_map.setdefault("getdatetime" , getdatetime)

				center_stack = ""
				left_stack   = ""
				ul_html      = ""
				html_str     = ""
				return dict_map
			else:
				print "---1__8\n"
				return dict_map
		else:
			print "----1_7\n"
			return dict_map
	else:
		print "----1_5\n"
		return dict_map



#获取 单个 app 详情
def get_App_details(itunesid , a_link , title , cul_typeid , cul_name , cul_pid , cul_pname):
	a_link   = a_link.replace("amp;" , "")
	curl     = initCurl()
	html_str = GetData(curl , a_link)

	if html_str:
		print cul_typeid + "-----" + a_link + "\n"
		return ___parsedata(html_str , title , a_link , itunesid , cul_typeid , cul_name , cul_pid , cul_pname)
	else:
		print a_link+" --- request faile \n"
		return ''

def check_Itunesid_isexists(itunesid):
	global conn
	cur  = conn.cursor()
	sql  = "select * from "+tb_name+" where itunesid='"+str(itunesid)+"' limit 1;"
	print sql
	cur.execute(sql)
	
	result  = []
	columns = tuple( [d[0].decode('utf8') for d in cur.description] )
	for row in cur:
	  result.append(dict(zip(columns, row))) 

	cur.close()
	return result

def isint(version):
	if version:
		if version.find(".") == -1:
			return True
		else:
			return False
	else:
		return True

def compare_version(html_version , db_version):
	compare = 0
	version1_list  = html_version.split(".")
	version2_list  = db_version.split(".")
	version1_count = len(version1_list)
	version2_count = len(version2_list)

	count = 0
	if version1_count >= version2_count:
		count = version1_count
	else:
		count = version2_count

	for i in xrange(0,count):
		if len(version1_list) > i:
			v1_node = version1_list[i]
		else:
			v1_node = 0

		if len(version2_list) > i:
			v2_node = version2_list[i]
		else:
			v2_node = 0

		if v1_node > v2_node:
			return True
		elif v1_node < v2_node :
			return False
	return False

def isNum(value):
    try:
        value + 1
    except TypeError:
        return False
    else:
        return True

def ____get_romte_info(itunesid , itunesDetail):

	global db_host
	global db_user
	global db_passwd
	global db_port
	global db_name
	global tb_name
	global conn
	global sqlite_mutex

	sqlite_mutex.acquire()

	itunesInfo = check_Itunesid_isexists(itunesid)
	if itunesInfo:
		# 更新 DB 
		db_version = itunesInfo[0].get("version")
		if isint(db_version):
			db_version = itunesInfo[0].get("bundleVersion")

		if compare_version(itunesDetail.get("version") , db_version):
			print "update : "+str(itunesid) + "----" + itunesInfo[0].get("version") + "===" + itunesDetail.get("version")+ "---"+itunesInfo[0].get("bundleVersion")
			update_sql = "update " + str(tb_name) + " set "
			update_where = ""
			field = 'title,senddate,apptypesno,score,fatherid,childid,itunes,seller,img,iphoneimg,ipadimg,equipmenttype,version,language,itunesprice,size,appupdate,system_m,isopen,system,content,getdatetime,newfeatures'
			field = field.split(",")
			for ff in field:
				# 通过 md5 对比
				tmp_value = str(itunesInfo[0].get(ff))
				m = hashlib.md5(tmp_value)
		        m.digest()
		        md5_info = m.hexdigest()

		        m = hashlib.md5(str(itunesDetail.get(ff)))
		        m.digest()
		        md5_detail = m.hexdigest()

		        if md5_info != md5_detail:
		        	# if ff == 'img':
		        	# 	print "=img="
		        	# if ff == 'iphoneimg':
		        	# 	print "=iphoneimg="
		        	# if ff == 'ipadimg':
		        	# 	print "=ipadimg="
		        	if isNum(itunesDetail.get(ff)) == False:
		        		if itunesDetail.get(ff) != None:
							tmp_value = itunesDetail.get(ff)
							tmp_value = str(tmp_value)
							tmp_value = MySQLdb.escape_string(tmp_value)
		        		else:
		        			tmp_value = itunesDetail.get(ff)
		        	else:
		        		tmp_value = itunesDetail.get(ff)

		        	if update_where:
		        		update_where = ','+str(ff) + "='"+str(tmp_value)+"'"
		        	else:
		        		update_where = str(ff) + "='"+str(tmp_value)+"'"

			# 开始 更新 DB
			if update_where :
				update_sql = update_sql + update_where + ",status = 3 where itunesid="+str(itunesid)+" limit 1;"
				cur  	   = conn.cursor()
				cur.execute(update_sql)
				conn.commit()
				cur.close()
		else:
			print "app " + str(itunesid) + " no update "
	else:
		# 插入 DB 
		# 下载所有图片

		itunesDetail.setdefault("status" , 2)
		itunesDetail.setdefault("is_genuine" , 1)
		itunesDetail.setdefault("is_hot" , 0)

		print itunesid + "----"+ itunesDetail.get("version")+"---insert\n"

		#新增 DB
		field = "itunesid,title,senddate,apptypesno,score,fatherid,childid,itunes,seller,img,iphoneimg,ipadimg,equipmenttype,version,language,itunesprice,size,appupdate,system_m,isopen,system,content,getdatetime,newfeatures,status,is_genuine,is_hot"
		field = field.split(",")
		insert_sql = ""
		insertBodySql = ""
		feild_str  = ''
		tmp_sql    = ""
		for ff in field:
			if isNum(itunesDetail.get(ff)) == False:
				if itunesDetail.get(ff) != None:
					tmp_value = itunesDetail.get(ff)
					tmp_value = str(tmp_value)
					tmp_value = MySQLdb.escape_string(tmp_value)
				else:
					tmp_value = itunesDetail.get(ff)	
			else:
				tmp_value = itunesDetail.get(ff)

			if tmp_sql:
				tmp_sql += ",'"+str(tmp_value)+"'"
			else:
				tmp_sql = "'"+str(tmp_value)+"'"

			if feild_str :
				feild_str += ",`"+str(ff)+"`"
			else:
				feild_str += "`"+str(ff)+"`"
		
		insertHeadsql = " INSERT INTO "+tb_name+" ("+feild_str+") VALUES";
		print "start insert sql ++++++++++++"
		if tmp_sql:

			if insertBodySql:
				insertBodySql = ",("+tmp_sql+")"
			else:
				insertBodySql = "("+tmp_sql+")"

			insert_sql = insertHeadsql + insertBodySql
			cur  = conn.cursor()
			cur.execute(insert_sql)
			conn.commit()
			cur.close()

	sqlite_mutex.release()





# name 类别名称 ， typeid 类别 id ， url 类别对应的 url ， 
def thread_main(cul_name , cul_typeid ,cul_pid , cul_pname , letter , pagenums , is_repeat):
	# 初始 url
	threadname = threading.currentThread().getName()
	print "Thread name:" + threadname +" Start "
	cul_url = "https://itunes.apple.com/WebObjects/MZStore.woa/wa/viewGenre?mt=8&letter="+str(letter)+"&page="+str(pagenums)+"&id="+str(cul_typeid)+"&cc=cn"
	datetime_now = datetime.datetime.now()
	datetime_now = datetime_now.strftime("%Y-%m-%d %H:%M")
	print " start "+cul_url + "----" + datetime_now + "\n"
	curl     = initCurl()
	html_str = GetData(curl , cul_url)
	is_have_url = 0
	if html_str:
		soup = BeautifulSoup(html_str)
		if soup.find(id="selectedcontent"):
			for obj_ul  in soup.find(id="selectedcontent").findAll('ul'):
				for obj_li in obj_ul.findAll('li'):
					a_link = obj_li.find('a').get("href").strip()
					print "======" + a_link + "::pagenums::" + str(pagenums) + "::: cul_typeid : " + str(cul_typeid) + ":::letter:"+str(letter)
					if a_link:
						a_name   = obj_li.find('a').getText()
						a_name   = a_name.replace("amp;" , '')
						itunesid = get_itunesid(a_link)
						if itunesid:
							is_have_url  = 1
							itunesDetail = get_App_details(itunesid , a_link , a_name , cul_typeid , cul_name , cul_pid , cul_pname)
							if itunesDetail and itunesDetail.has_key("version"):
								if(itunesDetail.get("isopen") == 1):
									print "not free app " + str(itunesDetail.get("itunesprice"))
									continue
								else:
									print "start parse single url"
									____get_romte_info(itunesid , itunesDetail)
									itunesDetail = ''
							else:
								time.sleep(5)
								itunesDetail = get_App_details(itunesid , a_link , a_name , cul_typeid , cul_name , cul_pid , cul_pname)
								if itunesDetail and itunesDetail.has_key("version"):
									if(itunesDetail.get("isopen") == 1):
										print "not free app " + str(itunesDetail.get("itunesprice"))
										continue
									else:
										print "start parse single url"
										____get_romte_info(itunesid , itunesDetail)
										itunesDetail = ''

		next_url  = ""
		totalPage = 0
		if soup.find('a' , {'class':'paginate-more'}):
			next_url = soup.find('a' , {'class':'paginate-more'}).get('href')
			next_url = next_url.replace("amp;" , '');
			print next_url
		else:
			totalPage = len(soup.find('ul' , {'class':'paginate'}).findAll('li'))

		html_str = ""
		# 抓取成功 开始进入下一页
		datetime_now = datetime.datetime.now()
		datetime_now = datetime_now.strftime("%Y-%m-%d %H:%M")
		if is_have_url:

			if next_url != '' or totalPage > pagenums:
				# next_url = 'https://itunes.apple.com/WebObjects/MZStore.woa/wa/viewGenre?mt=8&letter='+str(letter)+'&page='+pagenums+'&id='+cul_typeid+'&cc=cn'
				thread_main(cul_name , cul_typeid ,cul_pid , cul_pname , letter , pagenums , 0)
			else:
				print cul_typeid+"--"+cul_url+"---"+datetime_now+" --- Over \n";
				threads_nums = threads_nums - 1
				threadname = threading.currentThread().getName()
				print "Thread name:" + threadname +" end "
		else:#当前页面抓取失败

			commom_log('error_url_list.txt', cul_url+"\n" , write_type='a')

			if next_url != '' or totalPage > pagenums:
				if is_repeat == 1:
					pagenums += 1
					thread_main(cul_name , cul_typeid ,cul_pid , cul_pname , letter , pagenums , 0)
				else:
					time.sleep(1)
					thread_main(cul_name , cul_typeid ,cul_pid , cul_pname , letter , pagenums , 1)
			else:
				print cul_typeid+"--"+cul_url+"---"+datetime_now+" --- Over \n";
				threads_nums = threads_nums - 1
				threadname = threading.currentThread().getName()
				print "Thread name:" + threadname +" end "



def main():
	print 'start'
	
	global sqlite_mutex
	global threads_nums

	type_arr = [
		"图书|6018|https://itunes.apple.com/cn/genre/ios-tu-shu/id6018?mt=8|99999997|软件",
		"商业|6000|https://itunes.apple.com/cn/genre/ios-shang-ye/id6000?mt=8|99999997|软件",
		"商品指南|6022|https://itunes.apple.com/cn/genre/ios-shang-pin-zhi-nan/id6022?mt=8|99999997|软件",
		"教育|6017|https://itunes.apple.com/cn/genre/ios-jiao-yu/id6017?mt=8|99999997|软件",
		"娱乐|6016|https://itunes.apple.com/cn/genre/ios-yu-le/id6016?mt=8|99999997|软件",
		"财务|6015|https://itunes.apple.com/cn/genre/ios-cai-wu/id6015?mt=8|99999997|软件",
		"美食佳饮|6023|https://itunes.apple.com/cn/genre/ios-mei-shi-jia-yin/id6023?mt=8|99999997|软件",
		"健康健美|6013|https://itunes.apple.com/cn/genre/ios-jian-kang-jian-mei/id6013?mt=8|99999997|软件",
		"生活|6012|https://itunes.apple.com/cn/genre/ios-sheng-huo/id6012?mt=8|99999997|软件",
		"医疗|6020|https://itunes.apple.com/cn/genre/ios-yi-liao/id6020?mt=8|99999997|软件",
		"音乐|6011|https://itunes.apple.com/cn/genre/ios-yin-le/id6011?mt=8|99999997|软件",
		"导航|6010|https://itunes.apple.com/cn/genre/ios-dao-hang/id6010?mt=8|99999997|软件",
		"新闻|6009|https://itunes.apple.com/cn/genre/ios-xin-wen/id6009?mt=8|99999997|软件",
		"摄影与录像|6008|https://itunes.apple.com/cn/genre/ios-she-ying-yu-lu-xiang/id6008?mt=8|99999997|软件",
		"效率|6007|https://itunes.apple.com/cn/genre/ios-xiao-lu/id6007?mt=8|99999997|软件",
		"参考|6006|https://itunes.apple.com/cn/genre/ios-can-kao/id6006?mt=8|99999997|软件",
		"社交|6005|https://itunes.apple.com/cn/genre/ios-she-jiao/id6005?mt=8|99999997|软件",
		"体育|6004|https://itunes.apple.com/cn/genre/ios-ti-yu/id6004?mt=8|99999997|软件",
		"旅行|6003|https://itunes.apple.com/cn/genre/ios-lu-xing/id6003?mt=8|99999997|软件",
		"工具|6002|https://itunes.apple.com/cn/genre/ios-gong-ju/id6002?mt=8|99999997|软件",
		"天气|6001|https://itunes.apple.com/cn/genre/ios-tian-qi/id6001?mt=8|99999997|软件",

		# "动作游戏|7001|https://itunes.apple.com/cn/genre/ios-you-xi-dong-zuo-you-xi/id7001?mt=8|6014|游戏",
		# "探险游戏|7002|https://itunes.apple.com/cn/genre/ios-you-xi-tan-xian-you-xi/id7002?mt=8|6014|游戏",
		# "街机游戏|7003|https://itunes.apple.com/cn/genre/ios-you-xi-jie-ji-you-xi/id7003?mt=8|6014|游戏",
		# "桌面游戏|7004|https://itunes.apple.com/cn/genre/ios-you-xi-zhuo-mian-you-xi/id7004?mt=8|6014|游戏",
		# "扑克牌游戏|7005|https://itunes.apple.com/cn/genre/ios-you-xi-pu-ke-pai-you-xi/id7005?mt=8|6014|游戏",
		# "娱乐场游戏|7006|https://itunes.apple.com/cn/genre/ios-you-xi-yu-le-chang-you-xi/id7006?mt=8|6014|游戏",
		# "骰子游戏|7007|https://itunes.apple.com/cn/genre/ios-you-xi-tou-zi-you-xi/id7007?mt=8|6014|游戏",
		# "教育游戏|7008|https://itunes.apple.com/cn/genre/ios-you-xi-jiao-yu-you-xi/id7008?mt=8|6014|游戏",
		# "家庭游戏|7009|https://itunes.apple.com/cn/genre/ios-you-xi-jia-ting-you-xi/id7009?mt=8|6014|游戏",
		# "儿童游戏|7010|https://itunes.apple.com/cn/genre/ios-you-xi-er-tong-you-xi/id7010?mt=8|6014|游戏",
		# "音乐|7011|https://itunes.apple.com/cn/genre/ios-you-xi-yin-le/id7011?mt=8|6014|游戏",
		# "智力游戏|7012|https://itunes.apple.com/cn/genre/ios-you-xi-zhi-li-you-xi/id7012?mt=8|6014|游戏",
		# "赛车游戏|7013|https://itunes.apple.com/cn/genre/ios-you-xi-sai-che-you-xi/id7013?mt=8|6014|游戏",
		# "角色扮演游戏|7014|https://itunes.apple.com/cn/genre/ios-you-xi-jiao-se-ban-yan-you-xi/id7014?mt=8|6014|游戏",
		# "模拟游戏|7015|https://itunes.apple.com/cn/genre/ios-you-xi-mo-ni-you-xi/id7015?mt=8|6014|游戏",
		# "体育|7016|https://itunes.apple.com/cn/genre/ios-you-xi-ti-yu/id7016?mt=8|6014|游戏",
		# "策略游戏|7017|https://itunes.apple.com/cn/genre/ios-you-xi-ce-e-you-xi/id7017?mt=8|6014|游戏",
		# "文字游戏|7019|https://itunes.apple.com/cn/genre/ios-you-xi-wen-zi-you-xi/id7019?mt=8|6014|游戏",
		# "小游戏|7018|https://itunes.apple.com/cn/genre/ios-you-xi-xiao-you-xi/id7018?mt=8|6014|游戏",
	]
	letter_arr  = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']
	# letter_arr  = ['A']
	threads 	= []

	sqlite_mutex = threading.Lock()
	for x in type_arr:
		str_str = x.split("|")
		for l_x in letter_arr:
			print l_x
			threads.append(threading.Thread(target=thread_main, args=(str_str[0],str_str[1],str_str[3],str_str[4],l_x,1,0)))
	threads_nums = 0			
	for t in threads:
		if threads_nums < 500:
			t.start()
			t.join()
			threads_nums = threads_nums + 1
		else:
			time.sleep(10)
			print "sleep 10 s"

		

	print('end')



if __name__ == '__main__':
	reload(sys) 
	sys.setdefaultencoding('utf8')
	global db_host
	global db_user
	global db_passwd
	global db_port
	global db_name
	global tb_name
	global conn


	db_host   = "127.0.0.1"
	db_user   = 'root'
	db_passwd = ''
	db_port   = 3306
	db_name   = "app_xyapplist"
	tb_name   = "t_xyapplist_test"

	conn = MySQLdb.connect(host=db_host,user=db_user,passwd=db_passwd,port=db_port,init_command="set names utf8")
	conn.select_db(db_name)
	main()
	conn.close()

	# list_list = check_Itunesid_isexists(534966036)
	# print list_list[0].get("itunesid")

	# main()
	# cul_url  = 'https://itunes.apple.com/cn/app/tian-tian-ku-pao/id653350791?mt=8'
	# curl     = initCurl()
	# html_str = GetData(curl , cul_url) 
